#ifndef __PA_QWEN2_TOKENIZER_H__
#define __PA_QWEN2_TOKENIZER_H__

#include "common/common.h"

#include <string>

namespace pa
{

class Qwen2Tokenizer
{
public:
    Qwen2Tokenizer();
    ~Qwen2Tokenizer();

#ifndef PA_UNITTEST
private:
#endif // PA_UNITTEST
    std::u32string unicode_nfc_normalize(const std::string& prompt_origin);
    


};

} // namespace pa

#endif // __PA_QWEN2_TOKENIZER_H__